{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {},
   "source": [
    "## Exploring the company match dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {},
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: refuel-autolabel[openai] in /opt/homebrew/lib/python3.10/site-packages (0.0.1)\n",
      "Requirement already satisfied: loguru>=0.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.5.3)\n",
      "Requirement already satisfied: numpy>=1.23.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.23.3)\n",
      "Requirement already satisfied: requests>=2.27.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.28.1)\n",
      "Requirement already satisfied: datasets>=2.7.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2.11.0)\n",
      "Requirement already satisfied: langchain>=0.0.190 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.0.191)\n",
      "Requirement already satisfied: nervaluate>=0.1.8 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.1.8)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.5.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.1.2)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (8.2.2)\n",
      "Requirement already satisfied: SQLAlchemy==1.4.47 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.4.47)\n",
      "Requirement already satisfied: regex>=2023.6.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (2023.6.3)\n",
      "Requirement already satisfied: rich>=13.3.5 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (13.4.1)\n",
      "Requirement already satisfied: scipy>=1.10.1 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.1)\n",
      "Requirement already satisfied: pydantic>=1.10.9 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.10.9)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (1.13.0)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.6.0)\n",
      "Requirement already satisfied: wget>=3.2 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (3.2)\n",
      "Requirement already satisfied: openai>=0.27.4 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.27.6)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /opt/homebrew/lib/python3.10/site-packages (from refuel-autolabel[openai]) (0.3.3)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (9.0.0)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.3.5.1)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.64.1)\n",
      "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.70.13)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2022.10.0)\n",
      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (3.8.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.14.1)\n",
      "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (21.3)\n",
      "Requirement already satisfied: responses<0.19 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (0.18.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.10/site-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (6.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (0.5.7)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (2.8.4)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/homebrew/lib/python3.10/site-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (1.2.4)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.37.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/homebrew/lib/python3.10/site-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.10/site-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2022.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/homebrew/lib/python3.10/site-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.3.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.10/site-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.9.14)\n",
      "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/homebrew/lib/python3.10/site-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.15.1)\n",
      "Requirement already satisfied: joblib>=1.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/homebrew/lib/python3.10/site-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (22.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.8.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.3.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.10/site-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (1.2.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (3.19.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (1.5.1)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/homebrew/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.8.0)\n",
      "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets>=2.7.0->refuel-autolabel[openai]) (3.8.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/homebrew/lib/python3.10/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.12.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/homebrew/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai]) (0.4.3)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {},
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc181e31",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b17c64c1",
   "metadata": {},
   "source": [
    "#### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "66c63d82",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading seed example dataset to \"seed.csv\"...\n",
      "100% [........................................................] 474657 / 474657\n",
      "\n",
      "Downloading test dataset to \"test.csv\"...\n",
      "100% [......................................................] 4702697 / 4702697"
     ]
    }
   ],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data(\"company\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33c27634",
   "metadata": {},
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {},
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ababbe3d",
   "metadata": {},
   "source": [
    "### First labeling run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c93fae0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open(\"config_company_match.json\") as f:\n",
    "     config = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2448ffd6",
   "metadata": {},
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `entity_matching` (since it's an entity matching task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are provided with descriptions of companies from their websites...` (how we describe the task to the LLM)\n",
    "* `prompt.labels`: `['not duplicate', 'duplicate']` (the full list of labels to choose from)\n",
    "* `prompt.few_shot_num`: 3 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "31c4b617",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'CompanyEntityMatch',\n",
       " 'task_type': 'entity_matching',\n",
       " 'dataset': {'label_column': 'label', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are provided with descriptions of companies from their websites, and wikipedia pages. Your job is to categorize whether the descriptions are about the same company (duplicate) or different companies (not duplicate). Your answer must be from one of the following options:\\n{labels}',\n",
       "  'labels': ['not duplicate', 'duplicate'],\n",
       "  'example_template': 'Company 1 description: {entity1}\\nCompany 2 description: {entity2}\\nDuplicate or not: {label}',\n",
       "  'few_shot_examples': [{'entity1': 'lac wisconsin branding 95 1 & 96 1 the rock frequency 96.1 mhz translator s 95.1 w236ag fond du lac first air date 1965 as wcwc fm at 95.9 format mainstream rock erp 4 000 watts haat 123 meters 404 ft class a facility id 54510 transmitter coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 former callsigns wcwc fm 1965 1980 wyur 1980 1994 former frequencies 95.9 mhz 1965 affiliations cbs radio network westwood one premiere radio networks owner radio plus inc. sister stations wfdl wfdl fm wmdc webcast listen live website 961tcx . com studios in fond du lac wtcx 96.1 fm 95 1 & 96 1 the rock is a radio station broadcasting a mainstream rock music format . 1 licensed to ripon wisconsin usa the station is currently owned by radio plus inc. and features programing from cbs radio network dial global and premiere radio networks . 2 wtcx was originally on 95.9 mhz . be',\n",
       "    'entity2': 'closings contact next racing rocks local news breaking wiaa releases football playoffs matchups and brackets october 15 2016 local news here are the full brackets for the state of wisconsin division 1 2 seed fond du lac hosts 7 seed milwaukee washington friday october 21 at 7pm division 5 3 seed wla hosts 6 seed ... read more 10 15 16 fdl man injured in hit and run car vs. bike crash october 15 2016 local news a fond du lac man received non life threatening injuries in a car versus bicycle hit and run crash in dodge county . the dodge county sheriff s office says shortly after 8pm friday a car ... read more 10 15 16 ripon woman remains in critical condition following one vehicle crash october 15 2016 local news a ripon woman injured in a one vehicle crash after apparently falling asleep at the wheel remains in critical condition . the fond du lac county sheriff s office says 29 year old raquel amador ... read more wiaa releases football groupings october 15 2016 local news 2016 wiaa fo',\n",
       "    'label': 'duplicate'},\n",
       "   {'entity1': 'stacy spikes hamet watt headquarters new york city united states website http www.moviepass.com moviepass is a subscription based service for going to movie theaters available in the united states . the service gives members across the country the ability to see up to one 2d movie every 24 hours for a fixed monthly fee . members may choose which theaters they wish to attend and there are no blackout dates . moviepass works in nearly all movie theaters that accept the mastercard credit card making it one of the largest subscription based theater networks in america . prices vary by local market and start at 30 per month . moviepass was launched in february 2011 and is headquartered in new york city . 1 contents 1 service 2 purchasing a ticket 3 history 4 media coverage 5 references service edit the moviepass service works via a smartphone app iphone android and a specially designed reloadable debit card which is mailed to new subscribers when they sign up . purchasing a ticket edit in o',\n",
       "    'entity2': 'repair buy warranty get service buy warranty home warranty pricing & plans planning on moving home matters blog what s covered service professionals customer reviews benefits faqs appliance discount contract policies decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech close home warranty learn more what s covered service professionals faqs pricing and plans get a quote see plans planning on moving real estate plans buying a home selling a home home matters blog decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech our partner sites real estate professionals contractors 888 429 8247 email us log in back to top get a personalized quote explore plans in your area get covered in 3 easy steps . please correct highlighted fields request service log in create account oven on the fritz appliance breakdowns happen . get covered . get a personalized quote explore plans in your area get covered in 3 easy steps . please co',\n",
       "    'label': 'not duplicate'},\n",
       "   {'entity1': 'of over 110 gyms worldwide including 86 franchise locations in ma pa ny nj ct wa or ca tx fl ky va puerto rico and australia and is rapidly expanding across the u.s. and around the globe . contents 1 history 2 description 3 references 4 external links history edit crunch was founded in a basement level aerobics studio in new york city s east village in 1989 by doug levine . 1 with the collaboration of fitness instructors the group fitness programming was started at crunch . offerings such as hip hop aerobics co ed action wrestling and cyked yoga cycling were introduced . 2 in clubs members have access to innovative group fitness classes state of the art equipment personal and group training full service locker rooms and much more . select locations offer an exclusive crunch retail line that can also be purchased from the crunch online store . 3 in january 2014 crunch released its online workout extension called crunch live . this subscription based online video library has over 95 work',\n",
       "    'entity2': 'gallery esp en best rate guarantee check availability call us room only 1 800 990 8250 hotel air 1 800 219 2727 canada 1 855 478 2811 airport transportation travel agents close best rate guaranteebook your all inclusive stay hotel hotel air arrive departure adults 1 2 3 4 5 6 7 8 children 0 1 2 3 4 5 6 7 8 select property pacifica golf & spa resort the towers at pacifica sunset beach golf & spa resort ros resort & spa los cabos montecristo estates mazatl n emerald bay resort & spa emerald estates luxury villas departure country argentina australia austria bahamas belgium brazil canada chile colombia costa rica denmark ecuador finland france germany greece honduras iceland israel italy japan luxembourg mexico netherlands new zealand nicaragua norway panama paraguay peru portugal puerto rico republic of ireland republic of korea south africa spain sweden switzerland turks and caicos islands united kingdom united states uruguay venezuela departure city akron canton ohio reg . albany ny al',\n",
       "    'label': 'not duplicate'}],\n",
       "  'few_shot_selection': 'fixed',\n",
       "  'few_shot_num': 3}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "acb4a3de-fa84-4b94-b17a-7a6fac892a1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "92667a39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $12.172 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0061 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$12.172\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0061\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are provided with descriptions of companies from their websites, and wikipedia pages. Your job is to categorize whether the descriptions are about the same company (duplicate) or different companies (not duplicate). Your answer must be from one of the following options:\n",
      "not duplicate\n",
      "duplicate\n",
      "\n",
      "You will return the answer with one element: \"the correct option\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Company 1 description: lac wisconsin branding 95 1 & 96 1 the rock frequency 96.1 mhz translator s 95.1 w236ag fond du lac first air date 1965 as wcwc fm at 95.9 format mainstream rock erp 4 000 watts haat 123 meters 404 ft class a facility id 54510 transmitter coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 former callsigns wcwc fm 1965 1980 wyur 1980 1994 former frequencies 95.9 mhz 1965 affiliations cbs radio network westwood one premiere radio networks owner radio plus inc. sister stations wfdl wfdl fm wmdc webcast listen live website 961tcx . com studios in fond du lac wtcx 96.1 fm 95 1 & 96 1 the rock is a radio station broadcasting a mainstream rock music format . 1 licensed to ripon wisconsin usa the station is currently owned by radio plus inc. and features programing from cbs radio network dial global and premiere radio networks . 2 wtcx was originally on 95.9 mhz . be\n",
      "Company 2 description: closings contact next racing rocks local news breaking wiaa releases football playoffs matchups and brackets october 15 2016 local news here are the full brackets for the state of wisconsin division 1 2 seed fond du lac hosts 7 seed milwaukee washington friday october 21 at 7pm division 5 3 seed wla hosts 6 seed ... read more 10 15 16 fdl man injured in hit and run car vs. bike crash october 15 2016 local news a fond du lac man received non life threatening injuries in a car versus bicycle hit and run crash in dodge county . the dodge county sheriff s office says shortly after 8pm friday a car ... read more 10 15 16 ripon woman remains in critical condition following one vehicle crash october 15 2016 local news a ripon woman injured in a one vehicle crash after apparently falling asleep at the wheel remains in critical condition . the fond du lac county sheriff s office says 29 year old raquel amador ... read more wiaa releases football groupings october 15 2016 local news 2016 wiaa fo\n",
      "Duplicate or not: duplicate\n",
      "\n",
      "Company 1 description: stacy spikes hamet watt headquarters new york city united states website http www.moviepass.com moviepass is a subscription based service for going to movie theaters available in the united states . the service gives members across the country the ability to see up to one 2d movie every 24 hours for a fixed monthly fee . members may choose which theaters they wish to attend and there are no blackout dates . moviepass works in nearly all movie theaters that accept the mastercard credit card making it one of the largest subscription based theater networks in america . prices vary by local market and start at 30 per month . moviepass was launched in february 2011 and is headquartered in new york city . 1 contents 1 service 2 purchasing a ticket 3 history 4 media coverage 5 references service edit the moviepass service works via a smartphone app iphone android and a specially designed reloadable debit card which is mailed to new subscribers when they sign up . purchasing a ticket edit in o\n",
      "Company 2 description: repair buy warranty get service buy warranty home warranty pricing & plans planning on moving home matters blog what s covered service professionals customer reviews benefits faqs appliance discount contract policies decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech close home warranty learn more what s covered service professionals faqs pricing and plans get a quote see plans planning on moving real estate plans buying a home selling a home home matters blog decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech our partner sites real estate professionals contractors 888 429 8247 email us log in back to top get a personalized quote explore plans in your area get covered in 3 easy steps . please correct highlighted fields request service log in create account oven on the fritz appliance breakdowns happen . get covered . get a personalized quote explore plans in your area get covered in 3 easy steps . please co\n",
      "Duplicate or not: not duplicate\n",
      "\n",
      "Company 1 description: of over 110 gyms worldwide including 86 franchise locations in ma pa ny nj ct wa or ca tx fl ky va puerto rico and australia and is rapidly expanding across the u.s. and around the globe . contents 1 history 2 description 3 references 4 external links history edit crunch was founded in a basement level aerobics studio in new york city s east village in 1989 by doug levine . 1 with the collaboration of fitness instructors the group fitness programming was started at crunch . offerings such as hip hop aerobics co ed action wrestling and cyked yoga cycling were introduced . 2 in clubs members have access to innovative group fitness classes state of the art equipment personal and group training full service locker rooms and much more . select locations offer an exclusive crunch retail line that can also be purchased from the crunch online store . 3 in january 2014 crunch released its online workout extension called crunch live . this subscription based online video library has over 95 work\n",
      "Company 2 description: gallery esp en best rate guarantee check availability call us room only 1 800 990 8250 hotel air 1 800 219 2727 canada 1 855 478 2811 airport transportation travel agents close best rate guaranteebook your all inclusive stay hotel hotel air arrive departure adults 1 2 3 4 5 6 7 8 children 0 1 2 3 4 5 6 7 8 select property pacifica golf & spa resort the towers at pacifica sunset beach golf & spa resort ros resort & spa los cabos montecristo estates mazatl n emerald bay resort & spa emerald estates luxury villas departure country argentina australia austria bahamas belgium brazil canada chile colombia costa rica denmark ecuador finland france germany greece honduras iceland israel italy japan luxembourg mexico netherlands new zealand nicaragua norway panama paraguay peru portugal puerto rico republic of ireland republic of korea south africa spain sweden switzerland turks and caicos islands united kingdom united states uruguay venezuela departure city akron canton ohio reg . albany ny al\n",
      "Duplicate or not: not duplicate\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Company 1 description: lac wisconsin branding 95 1 & 96 1 the rock frequency 96.1 mhz translator s 95.1 w236ag fond du lac first air date 1965 as wcwc fm at 95.9 format mainstream rock erp 4 000 watts haat 123 meters 404 ft class a facility id 54510 transmitter coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 former callsigns wcwc fm 1965 1980 wyur 1980 1994 former frequencies 95.9 mhz 1965 affiliations cbs radio network westwood one premiere radio networks owner radio plus inc. sister stations wfdl wfdl fm wmdc webcast listen live website 961tcx . com studios in fond du lac wtcx 96.1 fm 95 1 & 96 1 the rock is a radio station broadcasting a mainstream rock music format . 1 licensed to ripon wisconsin usa the station is currently owned by radio plus inc. and features programing from cbs radio network dial global and premiere radio networks . 2 wtcx was originally on 95.9 mhz . before moving to 96.1 and was co owned with am 1600 wcwc now wrpn with studios in ripon wisconsin . previous logo edit references edit station information profile . arbitron . winter 2010 . retrieved 20\n",
      "Company 2 description: closings contact next racing rocks local news breaking wiaa releases football playoffs matchups and brackets october 15 2016 local news here are the full brackets for the state of wisconsin division 1 2 seed fond du lac hosts 7 seed milwaukee washington friday october 21 at 7pm division 5 3 seed wla hosts 6 seed ... read more 10 15 16 fdl man injured in hit and run car vs. bike crash october 15 2016 local news a fond du lac man received non life threatening injuries in a car versus bicycle hit and run crash in dodge county . the dodge county sheriff s office says shortly after 8pm friday a car ... read more 10 15 16 ripon woman remains in critical condition following one vehicle crash october 15 2016 local news a ripon woman injured in a one vehicle crash after apparently falling asleep at the wheel remains in critical condition . the fond du lac county sheriff s office says 29 year old raquel amador ... read more wiaa releases football groupings october 15 2016 local news 2016 wiaa football groupings times dates and actual matchups released on saturday afternoon . area teams in bold . division one group b seeding meeting host at fond du lac schools ... read more rock news you me a\n",
      "Duplicate or not: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dry-run -- this tells us how much this will cost and shows an example prompt\n",
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/company/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:41:08 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86846 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:08 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86846 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:10 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87975 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:10 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87975 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:11 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88881 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:11 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88881 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:12 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87253 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:12 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87253 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:15 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86849 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:15 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86849 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:17 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87584 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:17 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87584 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:18 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88590 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:18 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88590 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:19 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86962 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:19 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86962 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:22 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88732 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:22 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88732 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:24 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87083 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:24 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87083 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:41:27 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89084 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:27 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89084 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:28 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87451 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:28 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87451 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:30 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87006 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:30 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87006 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:32 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87908 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:32 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87908 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:33 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88820 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:33 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88820 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:34 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87194 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:35 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87194 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:38 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89075 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:38 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89075 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:39 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87444 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:39 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87444 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:41 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87007 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:41 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87007 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:43 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87983 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:43 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87983 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:41:44 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88966 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:44 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88966 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:45 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87320 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:45 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87320 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:49 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89109 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:49 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89109 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:50 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87475 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:50 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87475 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:52 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87037 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:52 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87037 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:54 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88036 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:54 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88036 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:55 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89082 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:55 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89082 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:56 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87437 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:56 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87437 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:41:59 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86934 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:41:59 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86934 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:00 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87937 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:00 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87937 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:42:02 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89023 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:02 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89023 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:03 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87390 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:03 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87390 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:05 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86852 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:05 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86852 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:07 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87578 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:07 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87578 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:09 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88677 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:09 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88677 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:10 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87022 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:10 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87022 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:13 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88989 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:13 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88989 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:14 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87335 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:14 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87335 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:17 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86784 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:17 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 86784 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:18 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87827 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:18 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87827 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:42:20 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88702 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:20 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88702 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:21 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87063 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:21 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87063 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:23 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89451 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:23 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 89451 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:25 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87818 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:25 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87818 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "2023-06-14 12:42:27 openai INFO: error_code=None error_message='Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87180 / min. Contact us through our help center at help.openai.com if you continue to have issues.' error_param=None error_type=tokens message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 12:42:27 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 87180 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Cost: 0.2862\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.87     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.87    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now, do the actual labeling\n",
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b82ab02",
   "metadata": {},
   "source": [
    "We are at 87% accuracy when labeling the first 100 examples. Let's see if we can use confidence scores to improve accuracy further by removing the less confident examples from our labeled set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {},
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6647796f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start computing confidence scores (using Refuel's LLMs)\n",
    "os.environ[\"REFUEL_API_KEY\"] = \"xxxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5fbc1264",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set `compute_confidence` -> True\n",
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1998f5e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "119e6f22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $12.172 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0061 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$12.172\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0061\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are provided with descriptions of companies from their websites, and wikipedia pages. Your job is to categorize whether the descriptions are about the same company (duplicate) or different companies (not duplicate). Your answer must be from one of the following options:\n",
      "not duplicate\n",
      "duplicate\n",
      "\n",
      "You will return the answer with one element: \"the correct option\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Company 1 description: lac wisconsin branding 95 1 & 96 1 the rock frequency 96.1 mhz translator s 95.1 w236ag fond du lac first air date 1965 as wcwc fm at 95.9 format mainstream rock erp 4 000 watts haat 123 meters 404 ft class a facility id 54510 transmitter coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 former callsigns wcwc fm 1965 1980 wyur 1980 1994 former frequencies 95.9 mhz 1965 affiliations cbs radio network westwood one premiere radio networks owner radio plus inc. sister stations wfdl wfdl fm wmdc webcast listen live website 961tcx . com studios in fond du lac wtcx 96.1 fm 95 1 & 96 1 the rock is a radio station broadcasting a mainstream rock music format . 1 licensed to ripon wisconsin usa the station is currently owned by radio plus inc. and features programing from cbs radio network dial global and premiere radio networks . 2 wtcx was originally on 95.9 mhz . be\n",
      "Company 2 description: closings contact next racing rocks local news breaking wiaa releases football playoffs matchups and brackets october 15 2016 local news here are the full brackets for the state of wisconsin division 1 2 seed fond du lac hosts 7 seed milwaukee washington friday october 21 at 7pm division 5 3 seed wla hosts 6 seed ... read more 10 15 16 fdl man injured in hit and run car vs. bike crash october 15 2016 local news a fond du lac man received non life threatening injuries in a car versus bicycle hit and run crash in dodge county . the dodge county sheriff s office says shortly after 8pm friday a car ... read more 10 15 16 ripon woman remains in critical condition following one vehicle crash october 15 2016 local news a ripon woman injured in a one vehicle crash after apparently falling asleep at the wheel remains in critical condition . the fond du lac county sheriff s office says 29 year old raquel amador ... read more wiaa releases football groupings october 15 2016 local news 2016 wiaa fo\n",
      "Duplicate or not: duplicate\n",
      "\n",
      "Company 1 description: stacy spikes hamet watt headquarters new york city united states website http www.moviepass.com moviepass is a subscription based service for going to movie theaters available in the united states . the service gives members across the country the ability to see up to one 2d movie every 24 hours for a fixed monthly fee . members may choose which theaters they wish to attend and there are no blackout dates . moviepass works in nearly all movie theaters that accept the mastercard credit card making it one of the largest subscription based theater networks in america . prices vary by local market and start at 30 per month . moviepass was launched in february 2011 and is headquartered in new york city . 1 contents 1 service 2 purchasing a ticket 3 history 4 media coverage 5 references service edit the moviepass service works via a smartphone app iphone android and a specially designed reloadable debit card which is mailed to new subscribers when they sign up . purchasing a ticket edit in o\n",
      "Company 2 description: repair buy warranty get service buy warranty home warranty pricing & plans planning on moving home matters blog what s covered service professionals customer reviews benefits faqs appliance discount contract policies decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech close home warranty learn more what s covered service professionals faqs pricing and plans get a quote see plans planning on moving real estate plans buying a home selling a home home matters blog decor cost savers lawn & garden lifestyle quick tips real estate repair & maintenance tech our partner sites real estate professionals contractors 888 429 8247 email us log in back to top get a personalized quote explore plans in your area get covered in 3 easy steps . please correct highlighted fields request service log in create account oven on the fritz appliance breakdowns happen . get covered . get a personalized quote explore plans in your area get covered in 3 easy steps . please co\n",
      "Duplicate or not: not duplicate\n",
      "\n",
      "Company 1 description: of over 110 gyms worldwide including 86 franchise locations in ma pa ny nj ct wa or ca tx fl ky va puerto rico and australia and is rapidly expanding across the u.s. and around the globe . contents 1 history 2 description 3 references 4 external links history edit crunch was founded in a basement level aerobics studio in new york city s east village in 1989 by doug levine . 1 with the collaboration of fitness instructors the group fitness programming was started at crunch . offerings such as hip hop aerobics co ed action wrestling and cyked yoga cycling were introduced . 2 in clubs members have access to innovative group fitness classes state of the art equipment personal and group training full service locker rooms and much more . select locations offer an exclusive crunch retail line that can also be purchased from the crunch online store . 3 in january 2014 crunch released its online workout extension called crunch live . this subscription based online video library has over 95 work\n",
      "Company 2 description: gallery esp en best rate guarantee check availability call us room only 1 800 990 8250 hotel air 1 800 219 2727 canada 1 855 478 2811 airport transportation travel agents close best rate guaranteebook your all inclusive stay hotel hotel air arrive departure adults 1 2 3 4 5 6 7 8 children 0 1 2 3 4 5 6 7 8 select property pacifica golf & spa resort the towers at pacifica sunset beach golf & spa resort ros resort & spa los cabos montecristo estates mazatl n emerald bay resort & spa emerald estates luxury villas departure country argentina australia austria bahamas belgium brazil canada chile colombia costa rica denmark ecuador finland france germany greece honduras iceland israel italy japan luxembourg mexico netherlands new zealand nicaragua norway panama paraguay peru portugal puerto rico republic of ireland republic of korea south africa spain sweden switzerland turks and caicos islands united kingdom united states uruguay venezuela departure city akron canton ohio reg . albany ny al\n",
      "Duplicate or not: not duplicate\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Company 1 description: lac wisconsin branding 95 1 & 96 1 the rock frequency 96.1 mhz translator s 95.1 w236ag fond du lac first air date 1965 as wcwc fm at 95.9 format mainstream rock erp 4 000 watts haat 123 meters 404 ft class a facility id 54510 transmitter coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 coordinates 43 49 10.00 n 88 43 20.00 w 43.8194444 n 88.7222222 w 43.8194444 ; 88.7222222 former callsigns wcwc fm 1965 1980 wyur 1980 1994 former frequencies 95.9 mhz 1965 affiliations cbs radio network westwood one premiere radio networks owner radio plus inc. sister stations wfdl wfdl fm wmdc webcast listen live website 961tcx . com studios in fond du lac wtcx 96.1 fm 95 1 & 96 1 the rock is a radio station broadcasting a mainstream rock music format . 1 licensed to ripon wisconsin usa the station is currently owned by radio plus inc. and features programing from cbs radio network dial global and premiere radio networks . 2 wtcx was originally on 95.9 mhz . before moving to 96.1 and was co owned with am 1600 wcwc now wrpn with studios in ripon wisconsin . previous logo edit references edit station information profile . arbitron . winter 2010 . retrieved 20\n",
      "Company 2 description: closings contact next racing rocks local news breaking wiaa releases football playoffs matchups and brackets october 15 2016 local news here are the full brackets for the state of wisconsin division 1 2 seed fond du lac hosts 7 seed milwaukee washington friday october 21 at 7pm division 5 3 seed wla hosts 6 seed ... read more 10 15 16 fdl man injured in hit and run car vs. bike crash october 15 2016 local news a fond du lac man received non life threatening injuries in a car versus bicycle hit and run crash in dodge county . the dodge county sheriff s office says shortly after 8pm friday a car ... read more 10 15 16 ripon woman remains in critical condition following one vehicle crash october 15 2016 local news a ripon woman injured in a one vehicle crash after apparently falling asleep at the wheel remains in critical condition . the fond du lac county sheriff s office says 29 year old raquel amador ... read more wiaa releases football groupings october 15 2016 local news 2016 wiaa football groupings times dates and actual matchups released on saturday afternoon . area teams in bold . division one group b seeding meeting host at fond du lac schools ... read more rock news you me a\n",
      "Duplicate or not: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/company/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "63c74705",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 12:51:22 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised APIConnectionError: Error communicating with OpenAI: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 13:07:02 openai INFO: error_code=None error_message='That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 7cc56551a9c8b91148d93eeecb5de640 in your message.)' error_param=None error_type=server_error message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:07:02 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 7cc56551a9c8b91148d93eeecb5de640 in your message.).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-06-14 13:20:49 openai INFO: error_code=None error_message='That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 14ddd2154ee03abc3a89a138b132c1f9 in your message.)' error_param=None error_type=server_error message='OpenAI API error received' stream_error=False\n",
      "2023-06-14 13:20:49 langchain.chat_models.openai WARNING: Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 14ddd2154ee03abc3a89a138b132c1f9 in your message.).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n",
       "</pre>\n"
      ],
      "text/plain": [
       "/opt/homebrew/lib/python3.10/site-packages/rich/live.py:231: UserWarning: install \"ipywidgets\" for Jupyter support\n",
       "  warnings.warn('install \"ipywidgets\" for Jupyter support')\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.5763\n",
      "Actual Cost: 1.6423\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 500     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.894    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9988    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.002           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 2       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9987    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.004           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 9       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9946    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8889   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.018           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 10      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9937    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.02            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 14      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9927    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8571   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.028           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 15      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9924    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.03            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 39      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9856    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9231   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.078           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 41      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9854    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.878    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.082           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 53      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9805    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9057   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.106           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 54      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9802    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8889   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.108           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 59      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9794    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8983   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.118           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 60      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9792    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8833   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.12            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 61      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9791    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8852   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.122           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 62      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.979     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.871    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.124           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 63      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.979     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.873    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.126           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 64      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9789    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8594   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.128           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 72      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9763    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.875    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.144           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 73      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9762    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.863    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.146           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 85      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.971     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8824   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.17            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 86      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9701    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8721   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.172           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 95      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9669    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8842   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.19            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 96      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9666    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.875    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.192           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 102     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9649    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8824   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.204           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 103     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9648    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8738   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.206           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 104     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9647    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.875    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.208           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 105     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9646    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8667   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.21            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 117     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9585    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8803   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.234           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 118     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.958     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8729   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.236           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 141     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9507    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8936   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.282           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 142     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9506    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8873   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.284           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 147     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9495    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8912   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.294           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 148     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9492    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8851   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.296           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 149     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9486    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8859   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.298           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 150     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9482    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.88     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.3             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 157     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9422    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8854   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.314           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 158     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9412    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8797   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.316           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 196     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.916     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9031   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.392           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 197     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9159    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8985   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.394           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 219     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.899     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9087   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.438           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 220     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8979    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9045   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.44            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 260     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8786    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9192   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.52            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 261     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8785    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9157   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.522           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 310     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8319    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.929    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.62            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 311     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8315    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.926    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.622           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 317     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8285    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9274   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.634           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 318     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8282    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9245   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.636           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 352     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7858    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9318   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.704           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 353     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.784     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9292   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.706           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 363     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7678    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9311   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.726           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 364     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7674    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9286   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.728           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 370     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7656    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9297   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.74            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 371     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7624    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9272   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.742           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 374     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7582    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9278   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.748           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 375     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.758     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9253   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.75            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 384     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7481    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9271   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.768           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 385     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7469    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9247   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.77            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 386     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7462    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9249   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.772           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 387     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7448    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9225   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.774           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 399     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7344    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9248   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.798           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 400     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7336    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9225   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 401     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7335    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9227   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.802           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 403     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7301    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9181   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.806           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 405     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7277    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9185   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.81            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 406     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7276    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9163   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.812           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 412     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7111    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9175   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.824           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 414     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7092    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.913    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.828           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 415     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7078    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9133   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.83            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 417     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7041    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9089   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.834           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 420     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7029    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9095   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.84            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 421     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7013    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9074   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.842           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 423     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6986    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9078   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.846           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 426     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6956    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9014   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.852           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 428     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6931    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9019   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.856           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 429     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6925    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8998   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.858           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 436     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6656    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9014   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.872           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 437     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6604    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8993   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.874           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 438     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6594    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8995   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.876           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 440     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6546    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8955   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.88            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 444     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6441    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8964   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.888           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 445     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6416    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8944   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 450     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6172    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8956   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 451     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6123    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8936   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.902           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 457     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5696    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.895    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.914           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 458     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5632    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.893    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.916           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 465     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5277    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8946   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.93            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 466     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5212    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8927   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.932           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 489     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.3397    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8978   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.978           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 490     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.3393    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8959   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.98            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 495     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.24      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.897    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.99            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 497     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.1575    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8934   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.994           </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 500     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0791    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.894    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m500    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.894   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9988   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.002          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m2      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9987   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.004          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m9      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9946   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8889  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.018          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m10     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9937   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.02           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m14     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9927   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8571  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.028          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m15     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9924   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.03           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m39     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9856   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9231  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.078          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m41     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9854   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.878   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.082          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m53     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9805   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9057  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.106          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m54     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9802   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8889  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.108          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m59     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9794   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8983  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.118          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m60     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9792   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8833  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.12           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m61     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9791   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8852  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.122          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m62     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.979    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.871   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.124          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m63     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.979    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.873   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.126          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m64     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9789   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8594  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.128          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m72     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9763   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.875   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.144          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m73     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9762   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.863   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.146          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m85     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.971    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8824  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.17           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m86     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9701   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8721  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.172          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m95     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9669   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8842  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.19           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m96     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9666   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.875   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.192          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m102    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9649   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8824  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.204          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m103    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9648   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8738  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.206          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m104    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9647   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.875   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.208          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m105    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9646   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8667  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.21           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m117    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9585   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8803  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.234          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m118    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.958    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8729  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.236          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m141    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9507   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8936  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.282          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m142    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9506   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8873  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.284          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m147    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9495   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8912  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.294          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m148    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9492   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8851  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.296          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m149    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9486   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8859  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.298          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m150    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9482   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.88    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.3            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m157    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9422   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8854  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.314          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m158    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9412   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8797  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.316          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m196    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.916    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9031  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.392          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m197    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9159   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8985  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.394          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m219    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.899    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9087  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.438          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m220    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8979   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9045  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.44           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m260    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8786   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9192  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.52           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m261    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8785   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9157  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.522          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m310    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8319   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.929   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.62           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m311    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8315   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.926   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.622          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m317    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8285   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9274  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.634          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m318    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8282   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9245  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.636          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m352    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7858   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9318  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.704          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m353    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.784    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9292  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.706          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m363    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7678   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9311  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.726          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m364    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7674   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9286  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.728          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m370    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7656   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9297  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.74           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m371    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7624   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9272  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.742          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m374    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7582   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9278  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.748          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m375    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.758    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9253  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.75           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m384    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7481   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9271  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.768          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m385    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7469   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9247  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.77           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m386    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7462   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9249  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.772          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m387    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7448   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9225  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.774          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m399    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7344   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9248  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.798          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m400    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7336   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9225  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m401    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7335   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9227  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.802          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m403    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7301   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9181  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.806          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m405    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7277   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9185  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.81           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m406    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7276   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9163  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.812          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m412    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7111   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9175  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.824          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m414    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7092   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.913   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.828          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m415    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7078   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9133  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.83           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m417    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7041   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9089  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.834          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m420    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7029   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9095  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.84           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m421    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7013   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9074  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.842          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m423    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6986   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9078  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.846          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m426    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6956   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9014  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.852          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m428    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6931   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9019  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.856          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m429    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6925   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8998  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.858          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m436    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6656   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9014  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.872          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m437    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6604   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8993  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.874          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m438    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6594   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8995  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.876          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m440    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6546   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8955  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.88           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m444    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6441   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8964  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.888          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m445    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6416   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8944  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m450    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6172   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8956  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m451    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6123   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8936  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.902          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m457    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5696   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.895   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.914          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m458    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5632   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.893   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.916          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m465    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5277   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8946  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.93           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m466    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5212   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8927  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.932          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m489    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.3397   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8978  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.978          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m490    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.3393   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8959  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.98           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m495    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.24     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.897   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.99           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m497    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.1575   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8934  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.994          \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m500    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0791   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.894   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2645e99",
   "metadata": {},
   "source": [
    "Looking at the table above, we can see that if we set the confidence threshold at `0.7656`, we are able to label at 93% accuracy and getting a completion rate of 74%. This means, we would ignore all the data points where confidence score is less than `0.7656` (which would end up being around 26% of all samples). This would, however, guarantee a very high quality labeled dataset for us. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
